import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import okhttp3.*;
import us.codecraft.webmagic.*;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Created by lkkjzx on 2017/4/25.
 */
public class SpiderProcessor implements PageProcessor {

    private OkHttpClient client = null;


    private Map<String, Object> tempMap = new HashMap<>();
    private String location=null;

    public SpiderProcessor(String location) {
        synchronized (this) {
            if (client == null) {
                client = new OkHttpClient();
            }
            this.location = location;
        }
    }
    private Site site = Site.me().setRetryTimes(2).setSleepTime(1500).setTimeOut(10000);
    private static String partyPageReg = "http://www.hdb.com/party/.*";
    private static String cityListNormalPageReg = "http://www.hdb.com/\\w*/0-0-2-0-\\d{1,3}";
//    private static String cityListNormalPageReg = "http://www.hdb.com/xiamen/0-0-2-0-\\d{1,3}";
    private static String partyUrlFormatStr="http://www.hdb.com/party/%s.html";
//    private static String cityPageReg="http://www.hdb.com/\\w*/?";

    private String getNowCityPartyListReg(){
        if(this.location==null){
            return "aaaaaaaaaa";
        }else {
            String cityPartyListStr="http://www.hdb.com/%s/0-0-2-0-\\d{1,3}";
            return String.format(cityPartyListStr, location);
        }
    }


    @Override
    public void process(Page page) {

        if(page.getStatusCode()!=200) return;

        if (page.getUrl().regex(partyPageReg).match()) {
            Selectable head = page.getHtml().xpath("//div[@class='content-body_head_r']");
            Selectable user = page.getHtml().xpath("//div[contains(@class,'detail_user')]");
            Party tempParty = new Party();
            //活动id
            String partyId = getPartyId(page.getUrl().toString());
            String title = head.xpath("//h1[@id='dt_title']/text()").toString();
            String url= page.getUrl().toString();
            String banner=page.getHtml().xpath("//div[@class='content-body_head_l']/img/@src").toString();

            String addr=head.xpath("//a[@class='detail_attr_blue']/text()").toString();
            //取不到 location
            //String location=getCity(addr);
            String longLatUrl=head.xpath("//a[@class='detail_attr_blue']/@href").toString();
            String[] longLatData = getLongLatNum(longLatUrl);
            String lon=longLatData[0];
            String lat=longLatData[1];
            String author = user.xpath("//a[@id='subinfo_name']/text()").toString();
            String authorUrl=user.xpath("//a[@id='subinfo_name']/@href").toString();
            String authorId=getAuthorId(authorUrl);
            String publishDay = user.xpath("//p[@class='fbTime']/text()").toString();
            String effectiveDay = head.xpath("//div[@class='detail_Time_t']/p/text()").toString();
            //有几种票,从javascript 中正则匹配
            String payItemString = page.getHtml().regex("_payItemListJson:\'.*?\'").toString();
            payItemString = payItemString==null?null:payItemString.substring(18, payItemString.length() - 1);
            //获取浏览次数和分享次数
            JSONObject hitsAndShareJsonObj = JSON.parseObject(getHitAndShareString(partyId));
            int hits= hitsAndShareJsonObj.getIntValue("hits");
            int share = hitsAndShareJsonObj.getIntValue("share");
            tempParty.setParty_id(partyId);
            tempParty.setTitle(filterEmoji(title));
            tempParty.setLocation(location);
            tempParty.setBanner(banner);
            tempParty.setHits(hits);
            tempParty.setShare(share);

            tempParty.setEffective_day(effectiveDay);
            tempParty.setPublish_day(publishDay);
            tempParty.setTicket_list(payItemString);
            tempParty.setUrl(url);

            tempParty.setAddr(filterEmoji(addr));
            tempParty.setLat(lat);
            tempParty.setLon(lon);

            tempParty.setAuthor(filterEmoji(author));
            tempParty.setAuthor_id(authorId);
            tempParty.setAuthor_url(authorUrl);

            //发布人详情
            JSONObject authorInfoObj=getAuthorInfo(authorId,partyId);
            if(authorInfoObj!=null&&authorId!=null){
                authorInfoObj.put("user_id",authorId);
                authorInfoObj.put("big_name",filterEmoji(author));
                page.putField("authorInfo",authorInfoObj);
            }
            //参加的用户
            JSONArray joinUserList=partyId == null ? new JSONArray() : getJoinPartyList(partyId, 1);

//            ArrayList<UserPartyMapper> joinPartyUserList = new ArrayList<>();
//            for (int i = 0; i < joinUserList.size(); i++) {
//                JSONObject tempJoinUserInfo = joinUserList.getJSONObject(i);
//                tempJoinUserInfo.put("party_id", partyId);
//            }
            int joinTotalNum=tempMap.containsKey("join_total_num")? (int) tempMap.get("join_total_num") :0;
            tempParty.setJoin_total_num(joinTotalNum);
            page.putField("partyInfo", tempParty);
            page.putField("joinUserInfo",joinUserList);


            //判断是否爬过这个作者了
//            有点慢而且被限制了
//            Reader reader=null;
//            SqlSession session=null;
//            try {
//                String resource = "mybatisConfig.xml";
//                reader = Resources.getResourceAsReader(resource);
//
//                SqlSessionFactory ssf = new SqlSessionFactoryBuilder().build(reader);
//                 session = ssf.openSession();
//                int authorCount=session.selectOne("com.twitso.interactSpider.AuthorProcess.select",authorId);
////                System.out.println("authorCount"+authorCount);
//                if(authorCount==0){
//                    ArrayList<String> nowPartyIdList=getAuthorPartyIdArray(1,authorId,1);
//                    ArrayList<String> completePartyIdList=getAuthorPartyIdArray(2,authorId,1);
//                    page.addTargetRequests(nowPartyIdList);
//                    page.addTargetRequests(completePartyIdList);
//                    session.insert("com.twitso.interactSpider.AuthorProcess.insert", authorId);
//                }
//            }catch (IOException e) {
//                e.printStackTrace();
//            }finally {
//                if (session != null) {
//                    session.close();
//                }
//                if (reader != null) {
//                    try {
//                        reader.close();
//                    } catch (IOException e) {
//                    }
//                }
//            }



        } else if (page.getUrl().regex(cityListNormalPageReg).match()) {
            //列表里面你加载
            page.addTargetRequests(page.getHtml().links().regex(getNowCityPartyListReg()).all());
//            page.addTargetRequest(page.getHtml().links().regex(partyPageReg).get());
            page.addTargetRequests(page.getHtml().links().regex(partyPageReg).all());
        }

    }




    @Override
    public Site getSite() {
        return site;
    }
    /**
     * @param url
     * @return partyId
     */
    private  String getPartyId(String url) {

        Matcher matcher = Pattern.compile("party/\\w{1,10}\\.html").matcher(url);
        if (matcher.find()) {
            String s = matcher.group();
            return s.substring(6, s.length() - 5);
        } else {
            return null;
        }
    }
    /**
     * @param author_url
     * @return author_id
     */
    private String getAuthorId(String author_url){
//        http://www.hdb.com/timeline/cam61.html
        if(author_url==null) return null;
        Matcher matcher=Pattern.compile("timeline/.*?\\.html").matcher(author_url);
        if(matcher.find()){
            String s=matcher.group();
            return s.substring(9,s.length()-5);
        }else {
            return null;
        }
    }
    private String[] getLongLatNum(String mapUrl){
        String[] longLatData = new String[2];
        if(mapUrl==null){
            return longLatData;
        }
        Matcher matcher=Pattern.compile("[\\d\\.]{3,}").matcher(mapUrl);
        int i=0;
        while (matcher.find()&&i<2){
            longLatData[i++] = matcher.group();
        }
        return longLatData;
    }


    private JSONObject getAuthorInfo(String authorId,String partyId){
        String url="http://www.hdb.com/query/api:124?user_id=%s&info_id=%s&info_type=party";
        okhttp3.Request request = new okhttp3.Request.Builder()
                .url(String.format(url, authorId,partyId))
                .build();
        Response response;
        try {
            response = client.newCall(request).execute();
            String json = response.body().string();
            return JSONObject.parseObject(json);
        } catch (IOException e) {
            return null;
        }
    }
    /**
     * 获取浏览和分享的jsonStr
     *
     * @param partyId
     * @return "{‘share’:123,‘hits’:1}"
     */
    private String getHitAndShareString(String partyId) {
        String url = "http://www.hdb.com/query/api:51?info_type=party&info_id=%s";
        okhttp3.Request request = new okhttp3.Request.Builder()
                .url(String.format(url, partyId))
                .build();
        Response response;
        try {
            response = client.newCall(request).execute();
            String json = response.body().string();
            return json;
        } catch (IOException e) {
            return "{\"share\":null,\"hits\":null}";
        }

    }

    /**
     * @param queryType 1-正在进行的活动 2-已经完成的游戏
     * @param authorId  发布者id
     * @param pageNum
     * @return partyIdArray belong this author e.g.{"ewnqb","jlnqb"}
     */
    private ArrayList<String> getAuthorPartyIdArray(int queryType, String authorId, int pageNum){
        String authorPartyTimeLineStr="http://www.hdb.com/post/api:170?query_type=%s&timeline_user_id36=%s&page_num=%s";
        ArrayList partyIdList = new ArrayList();
        okhttp3.Request request = new okhttp3.Request.Builder()
                .url(String.format(authorPartyTimeLineStr, queryType, authorId,pageNum))
                .build();
        Response response;
        String json;
        try {
            response = client.newCall(request).execute();
            json = response.body().string();
            //当前json的party数据
            Matcher partyIdMatcher=Pattern.compile("id36\":\"\\w+?\"").matcher(json);
            while (partyIdMatcher.find()){
                //id36":"ewnqb"
                String s=partyIdMatcher.group();
                partyIdList.add(String.format(partyUrlFormatStr,s.substring(7, s.length() - 1)));
            }
            //加载下一页数据
            Matcher hasNextPageMatcher = Pattern.compile("next_page[:\"]+1").matcher(json.substring(json.length() - 20));
            if(hasNextPageMatcher.find()){
                partyIdList.addAll(getAuthorPartyIdArray(queryType,authorId,++pageNum));
            }
        } catch (IOException e) {
        }finally {
            return partyIdList;
        }

    }

    private JSONArray getJoinPartyList(String partyId, int pageIndex) {

        String url = "http://www.hdb.com/query/api:53?party_id=%s&page_num=%s";

        okhttp3.Request request = new okhttp3.Request.Builder()
                .url(String.format(url, partyId, pageIndex))
                .build();
        Response response;
        String json;
        try {
            response = client.newCall(request).execute();
            json = response.body().string();

        } catch (IOException e) {
            return new JSONArray();
//            e.printStackTrace();
//            return
        }

        JSONObject jsonObj = JSON.parseObject(json);
        //报名参加人数
        if (pageIndex == 1) {
            tempMap.put("join_total_num", jsonObj.getInteger("join_total_num"));
        }
        if (jsonObj.containsKey("join_party_list")) {
            JSONArray JoinList = jsonObj.getJSONArray("join_party_list");
            if (jsonObj.getString("has_next_page").equals("1")) {
                JSONArray nextJsonArray = getJoinPartyList(partyId, ++pageIndex);
                JoinList.addAll(nextJsonArray);
            }
            return JoinList;
        } else {
            return new JSONArray();
        }
    }

    public static String filterEmoji(String source) {
        if(source!=null){
            return source.replaceAll("[\\ud800\\udc00-\\udbff\\udfff\\ud800-\\udfff]","");
        }else{
            return source;
        }
    }
}
